import json
import os
import random
import threading
import time

import facade
from xjlibrary.mdatetime.mtime import getTodayDate
from xjlibrary.mprocesspoll.MThreadingRun import MThreadingRun
from xjlibrary.mrequest.baserequest import USER_AGENTS
from xjlibrary.myredis.myredisclient import getDataFromRedis
from xjlibrary.our_file_dir import BaseDir

HEADERS = {
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "accept-encoding": "gzip, deflate, br",
    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
    "upgrade-insecure-requests": "1",
    "cache-control": "no-cache",
    "pragma": "no-cache",
    "referer": "https://www.jstor.org/action/showJournals?browseType=title",
    "user-agent": random.choice(USER_AGENTS)
}
BaseUrls = "https://www.jstor.org"

curPath = BaseDir.get_file_dir_absolute(__file__)
config_file = BaseDir.get_new_path(curPath, "db.ini")
TopPath = BaseDir.get_upper_dir(curPath, -2)
sPath = BaseDir.get_new_path(TopPath, "download", "jstorbook", "download", "book")
BaseDir.create_dir(sPath)
logger = facade.get_streamlogger()
mysqlutiles = facade.MysqlUtiles(config_file, "db", logger=logger)


def SelectFromDB():
    sql = "select url,bookname from book where stat=0 and failcount<3 limit 1000"
    return mysqlutiles.SelectFromDB(sql)


def readDBProxy():
    return getDataFromRedis(curPath, 'db.ini')


def down_book(threadval, url, bookname, list_proxy):
    result_queue = threadval.result_queue
    urlfull = BaseUrls + url
    # 伪造真实ip 防止被403
    HEADERS["X-Forwarded-For"] = list_proxy.get_random()
    BoolResult, errString, r = facade.MProxyRequest(urlfull,
                                                    HEADERS=HEADERS,
                                                    proxyRingList=list_proxy,
                                                    Feature='<div class="small-12 xlarge-9 columns mbm">',
                                                    timeout=(30, 60),
                                                    verify=False)
    if not BoolResult:
        result_queue.put((url, 0, 1))
        return
    outPathFile = BaseDir.get_new_path(sPath, '%d_%d.big_html' % (os.getpid(), threading.get_ident()))
    dicts = {}
    dicts["url"] = url
    dicts["bookname"] = bookname
    dicts["html"] = r.text
    dicts['downdate'] = getTodayDate()
    jsonmsg = json.dumps(dicts, ensure_ascii=False)
    BaseDir.single_add_file(outPathFile, jsonmsg + "\n")
    result_queue.put((url, 1, 0))


def InsertIntoDbFromList(results):
    liststat = []
    listFaile = []
    for url, stat, failcount in results:
        if stat == 1:
            liststat.append(url)
        else:
            listFaile.append(url)
    if len(liststat) >= 1:
        if len(liststat) == 1:
            liststat.append("test")
        sql = "update book set stat=1 where url in {}".format(tuple(liststat))
        mysqlutiles.ExeSqlToDB(sql, errExit=True)
    if len(listFaile) >= 1:
        if len(listFaile) == 1:
            listFaile.append("test")
        sql = "update book set failcount = failcount+1 where url in {}".format(tuple(listFaile))
        mysqlutiles.ExeSqlToDB(sql, errExit=True)


class DownBookThreadRun(MThreadingRun):

    def __init__(self, num):
        super(DownBookThreadRun, self).__init__(num)

    def getTask(self, *args, **kwargs):
        rows = SelectFromDB()
        return rows

    def setTask(self, results=None, *args, **kwargs):
        for url, bookname in results:
            self.add_job(self.func, url, bookname)

    def dealresult(self, *args, **kwargs):
        InsertIntoDbFromList(self.results)

    def setProxy(self, proxysList=None):
        list_proxy = []
        rows = readDBProxy()
        for proxy in rows:
            if proxy:
                list_proxy.append(proxy)
        print("proxy leng is :" + str(len(list_proxy)))
        MThreadingRun.setProxy(self, list_proxy)
        time.sleep(10)

    def is_break(self):
        return True

    def thread_pool_hook(self, thread_pool_dicts, thread, args, kwargs):
        return {}

    def fun(self, threadval, *args, **kwargs):
        url, bookname = args
        down_book(threadval, url, bookname, self.list_proxy)


def main():
    # func = down_book
    threadrun = DownBookThreadRun(40)
    threadrun.run()


if __name__ == "__main__":
    main()
